{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###对文本进行处理、、、、并提取基本的特征\n",
    "# 2021/3/10\n",
    "## by Hank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import os\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>studies show that you may be lied to every day...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>you could feel sorry for alberto torres , who ...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>although most weddings follow long-held tradit...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>it was 1961 and i was in the fifth grade . my ...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the sat is one of two major tests for the entr...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1402</th>\n",
       "      <td>come and see the india elephants and the new t...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1403</th>\n",
       "      <td>what are your clothes made of ? are they made ...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1404</th>\n",
       "      <td>most people do n't like mice , but they love o...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1405</th>\n",
       "      <td>val is a six-year-old boy . he begins to go to...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1406</th>\n",
       "      <td>my name is david and i have two brothers , mar...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1407 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                article   class\n",
       "0     studies show that you may be lied to every day...    high\n",
       "1     you could feel sorry for alberto torres , who ...    high\n",
       "2     although most weddings follow long-held tradit...    high\n",
       "3     it was 1961 and i was in the fifth grade . my ...    high\n",
       "4     the sat is one of two major tests for the entr...    high\n",
       "...                                                 ...     ...\n",
       "1402  come and see the india elephants and the new t...  middle\n",
       "1403  what are your clothes made of ? are they made ...  middle\n",
       "1404  most people do n't like mice , but they love o...  middle\n",
       "1405  val is a six-year-old boy . he begins to go to...  middle\n",
       "1406  my name is david and i have two brothers , mar...  middle\n",
       "\n",
       "[1407 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_raw = pd.read_csv(\"./data/test.csv\")\n",
    "test = test_raw.drop([\"Unnamed: 0\"],axis=1).copy()\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>my husband is a born shopper . he loves to loo...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tea drinking was common in china for nearly on...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>once upon a time , there was a scholar who wan...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>when newspapers and radio describe the damage ...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>below is a page adapted from about wikipedia o...</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25132</th>\n",
       "      <td>joe brown had a shop . he sold meat in his sho...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25133</th>\n",
       "      <td>if you do not use your arms or your legs for s...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25134</th>\n",
       "      <td>mrs. jones is waiting for an important telepho...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25135</th>\n",
       "      <td>new york is a very large city with 8 million p...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25136</th>\n",
       "      <td>there are 45 students in our class . i have ma...</td>\n",
       "      <td>middle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25137 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 article   class\n",
       "0      my husband is a born shopper . he loves to loo...    high\n",
       "1      tea drinking was common in china for nearly on...    high\n",
       "2      once upon a time , there was a scholar who wan...    high\n",
       "3      when newspapers and radio describe the damage ...    high\n",
       "4      below is a page adapted from about wikipedia o...    high\n",
       "...                                                  ...     ...\n",
       "25132  joe brown had a shop . he sold meat in his sho...  middle\n",
       "25133  if you do not use your arms or your legs for s...  middle\n",
       "25134  mrs. jones is waiting for an important telepho...  middle\n",
       "25135  new york is a very large city with 8 million p...  middle\n",
       "25136  there are 45 students in our class . i have ma...  middle\n",
       "\n",
       "[25137 rows x 2 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_raw = pd.read_csv(\"./data/train.csv\")\n",
    "train = train_raw.drop([\"Unnamed: 0\"],axis=1).copy()\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_article[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "【NLTK分词后去除停用词结果：】\n",
      "['studies', 'show', 'may', 'lied', 'every', 'day', 'anywhere', '10', '200', 'times', 'say', '``', 'nice', 'song', '``', '``', 'honey', \"n't\", 'look', 'fat', '``', 'another', 'study', 'showed', 'strangers', 'lied', 'three', 'times', 'within', 'first', '10', 'minutes', 'meeting', 'lie', 'strangers', 'lie', 'coworkers', 'men', 'lie', 'eight', 'times', 'people', 'women', 'lie', 'protect', 'people', \"'re\", 'married', \"'re\", 'going', 'lie', 'wife/', 'husband', 'one', 'every', '10', 'communications', \"'re\", 'unmarried', 'number', 'drops', 'three', 'look', 'point', 'got', 'lied', \"'s\", 'agreed', 'get', 'lied', 'truth', 'lying', 'lying', \"'s\", 'cooperative', 'act', 'lies', 'harmful', 'sometimes', \"'re\", 'willing', 'lie', 'sake', 'social', 'dignity', 'maybe', 'keep', 'private', 'secret', 'lying', 'complex', \"'s\", 'woven', 'fabric', 'daily', 'business', 'lives', \"'re\", 'deeply', 'disturbed', 'truth', 'explain', 'sometimes', 'good', 'reasons', 'times', \"n't\", 'understand', 'gaps', 'ideals', 'realities', 'lives', \"'re\", 'lying', 'secretly', \"'re\", 'ways', 'society', 'practiced', 'centuries', 'centuries', \"'s\", 'old', 'breathing', \"'s\", 'part', 'culture', 'history', 'think', 'stories', 'dante', 'shakespeare', 'bible', 'news', 'world', 'lying', 'great', 'value', 'evolution', 'human', 'researchers', 'long', 'known', 'intelligent', 'species', 'likely', 'lie', 'human', 'like', 'become', 'leaders', 'starts', 'really', 'early', 'early', 'well', 'babies', 'pretend', 'cry', 'pause', 'wait', 'see', \"'s\", 'coming', 'go', 'right', 'back', 'crying', 'one-year-olds', 'learn', 'hiding', 'truth', 'two-year-olds', 'bluff', 'five-year-olds', 'lie', 'outright', 'try', 'control', 'via', 'flattery', 'nine-year-olds', 'masters', 'covering', 'time', 'enter', 'college', \"'re\", 'going', 'lie', 'mom', 'one', 'every', 'five', 'interactions', 'time', 'enter', 'work', 'world', 'breadwinners', \"'ve\", 'entered', 'world', 'full', 'fake', 'digital', 'friends', 'business', 'media', 'identity', 'thieves', 'world-class', 'cheats', '--', '--', 'short', 'post-truth', 'society', 'well', 'steps', 'take', 'guide', 'way', 'bushes', 'trained', 'lie', 'spotters', 'get', 'truth', '90', 'time', 'rest', 'us', \"'re\", '54', 'accurate', 'easy', 'learn', 'clever', 'liars', 'stupid', 'liars', \"'re\", 'real', 'original', 'liars', 'make', 'mistakes', 'use', 'techniques']\n",
      "\n",
      "【NLTK分词进行词干提取：】\n",
      "['studi', 'show', 'may', 'lie', 'everi', 'day', 'anywher', '10', '200', 'time', 'say', '``', 'nice', 'song', '``', '``', 'honey', \"n't\", 'look', 'fat', '``', 'anoth', 'studi', 'show', 'stranger', 'lie', 'three', 'time', 'within', 'first', '10', 'minut', 'meet', 'lie', 'stranger', 'lie', 'cowork', 'men', 'lie', 'eight', 'time', 'peopl', 'women', 'lie', 'protect', 'peopl', \"'re\", 'marri', \"'re\", 'go', 'lie', 'wife/', 'husband', 'one', 'everi', '10', 'commun', \"'re\", 'unmarri', 'number', 'drop', 'three', 'look', 'point', 'got', 'lie', \"'s\", 'agre', 'get', 'lie', 'truth', 'lie', 'lie', \"'s\", 'cooper', 'act', 'lie', 'harm', 'sometim', \"'re\", 'will', 'lie', 'sake', 'social', 'digniti', 'mayb', 'keep', 'privat', 'secret', 'lie', 'complex', \"'s\", 'woven', 'fabric', 'daili', 'busi', 'live', \"'re\", 'deepli', 'disturb', 'truth', 'explain', 'sometim', 'good', 'reason', 'time', \"n't\", 'understand', 'gap', 'ideal', 'realiti', 'live', \"'re\", 'lie', 'secretli', \"'re\", 'way', 'societi', 'practic', 'centuri', 'centuri', \"'s\", 'old', 'breath', \"'s\", 'part', 'cultur', 'histori', 'think', 'stori', 'dant', 'shakespear', 'bibl', 'news', 'world', 'lie', 'great', 'valu', 'evolut', 'human', 'research', 'long', 'known', 'intellig', 'speci', 'like', 'lie', 'human', 'like', 'becom', 'leader', 'start', 'realli', 'earli', 'earli', 'well', 'babi', 'pretend', 'cri', 'paus', 'wait', 'see', \"'s\", 'come', 'go', 'right', 'back', 'cri', 'one-year-old', 'learn', 'hide', 'truth', 'two-year-old', 'bluff', 'five-year-old', 'lie', 'outright', 'tri', 'control', 'via', 'flatteri', 'nine-year-old', 'master', 'cover', 'time', 'enter', 'colleg', \"'re\", 'go', 'lie', 'mom', 'one', 'everi', 'five', 'interact', 'time', 'enter', 'work', 'world', 'breadwinn', \"'ve\", 'enter', 'world', 'full', 'fake', 'digit', 'friend', 'busi', 'media', 'ident', 'thiev', 'world-class', 'cheat', '--', '--', 'short', 'post-truth', 'societi', 'well', 'step', 'take', 'guid', 'way', 'bush', 'train', 'lie', 'spotter', 'get', 'truth', '90', 'time', 'rest', 'us', \"'re\", '54', 'accur', 'easi', 'learn', 'clever', 'liar', 'stupid', 'liar', \"'re\", 'real', 'origin', 'liar', 'make', 'mistak', 'use', 'techniqu']\n"
     ]
    }
   ],
   "source": [
    "test_article = test[\"article\"]\n",
    "test_class = test[\"class\"]\n",
    "words = nltk.word_tokenize(test_article[0])\n",
    "#words\n",
    "interpunctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%']   #定义标点符号列表\n",
    "cutwords1 = [word for word in words if word not in interpunctuations] \n",
    "\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "stops = set(stopwords.words(\"english\"))\n",
    "cutwords2 = [word for word in cutwords1 if word not in stops]  #判断分词在不在停用词列表内\n",
    "print('\\n【NLTK分词后去除停用词结果：】')\n",
    "print(cutwords2)\n",
    "\n",
    "print('\\n【NLTK分词进行词干提取：】')\n",
    "cutwords3 = []\n",
    "for cutword in cutwords2:\n",
    "    cutwords3.append(PorterStemmer().stem(cutword))    #词干提取\n",
    "print(cutwords3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\" '' `` honey , you do n't look fat in that , no \""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_article[0].split(\".\")[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Packages\n",
    "import nltk \n",
    "import pandas as pd\n",
    "import glob\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tag import pos_tag\n",
    "from collections import Counter\n",
    "from nltk import sent_tokenize\n",
    "# nltk.download('wordnet')\n",
    "# nltk.download('stopwords')\n",
    "# nltk.download('averaged_perceptron_tagger')\n",
    "    \n",
    "class features:\n",
    "    Dale_Chall_List = pd.read_csv(\"./data/Dale Chall List.txt\")\n",
    "    columns = ['Class','Total Number Of Sentences', 'Average Sentence Length', 'Average Word Length', \\\n",
    "               'Number of Uncommon Words', 'Number of Unique Words', 'Words with 1 to 3 syllables', \\\n",
    "               \"Words with 4 syllables\", \"Words with 5 syllables\", \"Words with 6 syllables\", \\\n",
    "               \"Words with more than 7 syllables\", \"Average number of syllables\"]\n",
    "    ptcol = ['CC', 'CD', 'NNS', 'VBP', 'NN', 'RB', 'MD', 'VB', 'VBZ', 'VBD', 'VBG', 'IN', 'JJ',  'FW', 'WDT', \\\n",
    "     'RBR', 'PRP$', 'VBN', 'PRP', 'DT', 'JJS', 'RP', 'JJR', 'WRB',  'WP', 'NNP', 'WP$',\\\n",
    "     'PDT', 'RBS', \"''\", 'NNPS', 'SYM', 'EX','TO','UH']\n",
    "    ptcol.sort()\n",
    "    columns.extend(ptcol)\n",
    "\n",
    "#Preprocessing\n",
    "    def preprocessing(self,text1):    \n",
    "        text1 = re.sub('[^a-zA-Z]', ' ', text1)\n",
    "        return [word for word in text1.lower().split() if not word in set(stopwords.words('english'))]\n",
    " \n",
    "#Feature extraction\n",
    "    def avg_sentence_length(self, text, num_sents):\n",
    "        \"\"\"句子平均长度\"\"\"\n",
    "        avg = float(len(text)/num_sents)\n",
    "        return avg\n",
    " \n",
    "    def avg_word_length(self, text):\n",
    "        \"\"\"单词的平均长度\"\"\"\n",
    "        s=0\n",
    "        for w in text:\n",
    "            s+=len(w)\n",
    "            a=s/len(text)\n",
    "        return a\n",
    " \n",
    "    def syllable_count_single_word(self, word):\n",
    "        \"\"\"单音节的总数\"\"\"\n",
    "        word = word.lower()\n",
    "        count = 0\n",
    "        vowels = \"aeiouy\"\n",
    "        if word[0] in vowels:\n",
    "            count += 1\n",
    "        for index in range(1, len(word)):\n",
    "            if word[index] in vowels and word[index - 1] not in vowels:\n",
    "                count += 1\n",
    "        if word.endswith(\"e\"):\n",
    "            count -= 1\n",
    "        if count == 0:\n",
    "            count += 1\n",
    "        return count\n",
    " \n",
    "    def avg_syllables(self, text):\n",
    "        \"\"\"文章单音节\"\"\"\n",
    "        s=0\n",
    "        for w in text:\n",
    "            s+=self.syllable_count_single_word(w)\n",
    "        a=s/len(text)\n",
    "        return a\n",
    " \n",
    "    def pos_count_in_list(self, list1):\n",
    "        \"\"\"词性标注\"\"\"\n",
    "        pt = pos_tag(list1)\n",
    "        dictpt = dict(pt)\n",
    "        dictpt = Counter(dictpt.values())\n",
    "        vl = []\n",
    "        for i in self.ptcol:\n",
    "            if i in dictpt.keys():\n",
    "                vl.append(dictpt[i])\n",
    "            else:\n",
    "                vl.append(0)\n",
    "        return vl\n",
    " \n",
    "    def dif_words(self, text):\n",
    "        \"\"\"词频\"\"\"\n",
    "        frequency = Counter(text)\n",
    "        return len(frequency)\n",
    " \n",
    "    def freq_syl(self, text):\n",
    "        \"\"\"平均音节频数\"\"\"\n",
    "        count = [0,0,0,0,0]\n",
    "        uniq_words = Counter(text).keys()\n",
    "        for word in uniq_words:\n",
    "            x = self.syllable_count_single_word(word)\n",
    "            if(x > 1 and x <=3):\n",
    "                count[0]+=1\n",
    "            elif(x == 4):\n",
    "                count[1]+=1\n",
    "            elif(x == 5):\n",
    "                count[2]+=1\n",
    "            elif(x == 6):\n",
    "                count[3]+=1\n",
    "            else:\n",
    "                count[4]+=1\n",
    "        return count\n",
    " \n",
    "    def not_in_dale_chall(self,text):\n",
    "        \"\"\"戴尔查词表\"\"\"\n",
    "        n = [w for w in text if w not in self.Dale_Chall_List]\n",
    "        n1 = len(n)\n",
    "        return n1\n",
    "    \n",
    "#     def total_polysyllables(self):\n",
    "#         \"\"\"多音节总数\"\"\"\n",
    "#         return\n",
    "    \n",
    "#     def total_words(self):\n",
    "#         \"\"\"总词数\"\"\"\n",
    "#         return len(nltk.word_tokenize(self))\n",
    "    \n",
    "    \n",
    "\n",
    "#CREATING THE DATAFRAMES\n",
    "    def fextr(self,txt_article,txt_class):\n",
    "        c2 = [txt_class]\n",
    "#         file = open(filename, 'r', encoding = \"utf8\")\n",
    "#         text2 = file.read()\n",
    "        text2 = txt_article\n",
    "        txt = self.preprocessing(text2)\n",
    "        #print(txt,len(txt))\n",
    "        avg1 = len(sent_tokenize(text2))#句子总数\n",
    "        c2.append(avg1) \n",
    "        c2.append(self.avg_sentence_length(txt, avg1))\n",
    "        c2.extend([self.avg_word_length(txt),self.not_in_dale_chall(txt),self.dif_words(txt)])\n",
    "        c2.extend(self.freq_syl(txt))\n",
    "        c2.append(self.avg_syllables(txt))\n",
    "        vallist = self.pos_count_in_list(txt)\n",
    "        c2.extend(vallist)\n",
    "        #print(len(c2),c2)\n",
    "        return c2\n",
    "    \n",
    "    def create_dataframe(self,txt):\n",
    "        data = []       \n",
    "        for index, row in txt.iterrows():\n",
    "            #print(type(np.array(row[\"class\"])),(list(self.fextr(row[\"article\"]))))\n",
    "            data.append(list(self.fextr(row[\"article\"],row[\"class\"])))\n",
    "            #print(row[\"article\"], row[\"class\"])\n",
    "            #print(type(row[\"article\"]), type(row[\"class\"]))\n",
    "            #print(data)\n",
    "            if index%1000 == 0:\n",
    "                print(\"当前已处理文本数量为：\",index)\n",
    "                #print(\"当前处理文本为：\",data)\n",
    "        #data = np.array(data).reshape(len(data),47)\n",
    "        df = pd.DataFrame(data,columns = self.columns)\n",
    "        return(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Total Number Of Sentences</th>\n",
       "      <th>Average Sentence Length</th>\n",
       "      <th>Average Word Length</th>\n",
       "      <th>Number of Uncommon Words</th>\n",
       "      <th>Number of Unique Words</th>\n",
       "      <th>Words with 1 to 3 syllables</th>\n",
       "      <th>Words with 4 syllables</th>\n",
       "      <th>Words with 5 syllables</th>\n",
       "      <th>Words with 6 syllables</th>\n",
       "      <th>...</th>\n",
       "      <th>VB</th>\n",
       "      <th>VBD</th>\n",
       "      <th>VBG</th>\n",
       "      <th>VBN</th>\n",
       "      <th>VBP</th>\n",
       "      <th>VBZ</th>\n",
       "      <th>WDT</th>\n",
       "      <th>WP</th>\n",
       "      <th>WP$</th>\n",
       "      <th>WRB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>high</td>\n",
       "      <td>42</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>231</td>\n",
       "      <td>173</td>\n",
       "      <td>81</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>high</td>\n",
       "      <td>45</td>\n",
       "      <td>7.622222</td>\n",
       "      <td>5.253644</td>\n",
       "      <td>343</td>\n",
       "      <td>231</td>\n",
       "      <td>100</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>high</td>\n",
       "      <td>24</td>\n",
       "      <td>6.708333</td>\n",
       "      <td>6.279503</td>\n",
       "      <td>161</td>\n",
       "      <td>124</td>\n",
       "      <td>58</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>high</td>\n",
       "      <td>35</td>\n",
       "      <td>7.571429</td>\n",
       "      <td>5.418868</td>\n",
       "      <td>265</td>\n",
       "      <td>193</td>\n",
       "      <td>99</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>high</td>\n",
       "      <td>22</td>\n",
       "      <td>7.545455</td>\n",
       "      <td>6.373494</td>\n",
       "      <td>166</td>\n",
       "      <td>115</td>\n",
       "      <td>65</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1402</th>\n",
       "      <td>middle</td>\n",
       "      <td>6</td>\n",
       "      <td>7.500000</td>\n",
       "      <td>5.066667</td>\n",
       "      <td>45</td>\n",
       "      <td>41</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1403</th>\n",
       "      <td>middle</td>\n",
       "      <td>28</td>\n",
       "      <td>4.785714</td>\n",
       "      <td>5.007463</td>\n",
       "      <td>134</td>\n",
       "      <td>88</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1404</th>\n",
       "      <td>middle</td>\n",
       "      <td>13</td>\n",
       "      <td>5.846154</td>\n",
       "      <td>5.421053</td>\n",
       "      <td>76</td>\n",
       "      <td>46</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1405</th>\n",
       "      <td>middle</td>\n",
       "      <td>18</td>\n",
       "      <td>4.111111</td>\n",
       "      <td>4.851351</td>\n",
       "      <td>74</td>\n",
       "      <td>50</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1406</th>\n",
       "      <td>middle</td>\n",
       "      <td>6</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>5.034483</td>\n",
       "      <td>29</td>\n",
       "      <td>19</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1407 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Class  Total Number Of Sentences  Average Sentence Length  \\\n",
       "0       high                         42                 5.500000   \n",
       "1       high                         45                 7.622222   \n",
       "2       high                         24                 6.708333   \n",
       "3       high                         35                 7.571429   \n",
       "4       high                         22                 7.545455   \n",
       "...      ...                        ...                      ...   \n",
       "1402  middle                          6                 7.500000   \n",
       "1403  middle                         28                 4.785714   \n",
       "1404  middle                         13                 5.846154   \n",
       "1405  middle                         18                 4.111111   \n",
       "1406  middle                          6                 4.833333   \n",
       "\n",
       "      Average Word Length  Number of Uncommon Words  Number of Unique Words  \\\n",
       "0                5.454545                       231                     173   \n",
       "1                5.253644                       343                     231   \n",
       "2                6.279503                       161                     124   \n",
       "3                5.418868                       265                     193   \n",
       "4                6.373494                       166                     115   \n",
       "...                   ...                       ...                     ...   \n",
       "1402             5.066667                        45                      41   \n",
       "1403             5.007463                       134                      88   \n",
       "1404             5.421053                        76                      46   \n",
       "1405             4.851351                        74                      50   \n",
       "1406             5.034483                        29                      19   \n",
       "\n",
       "      Words with 1 to 3 syllables  Words with 4 syllables  \\\n",
       "0                              81                       7   \n",
       "1                             100                      11   \n",
       "2                              58                       6   \n",
       "3                              99                       4   \n",
       "4                              65                       7   \n",
       "...                           ...                     ...   \n",
       "1402                           16                       1   \n",
       "1403                           26                       1   \n",
       "1404                           24                       2   \n",
       "1405                           20                       1   \n",
       "1406                            8                       0   \n",
       "\n",
       "      Words with 5 syllables  Words with 6 syllables  ...  VB  VBD  VBG  VBN  \\\n",
       "0                          1                       0  ...   3    8    8    2   \n",
       "1                          2                       0  ...  13   15   12    6   \n",
       "2                          2                       0  ...   4    3    6    3   \n",
       "3                          1                       0  ...   6   22    8    3   \n",
       "4                          6                       0  ...   3    4    4    6   \n",
       "...                      ...                     ...  ...  ..  ...  ...  ...   \n",
       "1402                       0                       0  ...   3    0    1    1   \n",
       "1403                       0                       0  ...   4   10    1    0   \n",
       "1404                       0                       0  ...   0    1    1    3   \n",
       "1405                       0                       0  ...   1    7    1    0   \n",
       "1406                       0                       0  ...   0    0    0    0   \n",
       "\n",
       "      VBP  VBZ  WDT  WP  WP$  WRB  \n",
       "0      13    4    0   0    0    0  \n",
       "1       8    8    1   0    0    0  \n",
       "2      13    1    0   0    0    0  \n",
       "3       8    4    0   0    0    0  \n",
       "4       6    4    0   0    0    0  \n",
       "...   ...  ...  ...  ..  ...  ...  \n",
       "1402    4    0    0   0    0    0  \n",
       "1403    4    0    0   1    0    0  \n",
       "1404    2    2    0   0    0    0  \n",
       "1405    3    3    0   0    0    0  \n",
       "1406    1    1    0   0    0    0  \n",
       "\n",
       "[1407 rows x 47 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_features.to_csv(\"./data/test_features.csv\", encoding=\"utf-8\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当前已处理文本数量为： 0\n",
      "当前已处理文本数量为： 1000\n",
      "当前已处理文本数量为： 2000\n",
      "当前已处理文本数量为： 3000\n",
      "当前已处理文本数量为： 4000\n",
      "当前已处理文本数量为： 5000\n",
      "当前已处理文本数量为： 6000\n",
      "当前已处理文本数量为： 7000\n",
      "当前已处理文本数量为： 8000\n",
      "当前已处理文本数量为： 9000\n",
      "当前已处理文本数量为： 10000\n",
      "当前已处理文本数量为： 11000\n",
      "当前已处理文本数量为： 12000\n",
      "当前已处理文本数量为： 13000\n",
      "当前已处理文本数量为： 14000\n",
      "当前已处理文本数量为： 15000\n",
      "当前已处理文本数量为： 16000\n",
      "当前已处理文本数量为： 17000\n",
      "当前已处理文本数量为： 18000\n",
      "当前已处理文本数量为： 19000\n",
      "当前已处理文本数量为： 20000\n",
      "当前已处理文本数量为： 21000\n",
      "当前已处理文本数量为： 22000\n",
      "当前已处理文本数量为： 23000\n",
      "当前已处理文本数量为： 24000\n",
      "当前已处理文本数量为： 25000\n"
     ]
    }
   ],
   "source": [
    "f2 = features()\n",
    "text_features = f2.create_dataframe(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Total Number Of Sentences</th>\n",
       "      <th>Average Sentence Length</th>\n",
       "      <th>Average Word Length</th>\n",
       "      <th>Number of Uncommon Words</th>\n",
       "      <th>Number of Unique Words</th>\n",
       "      <th>Words with 1 to 3 syllables</th>\n",
       "      <th>Words with 4 syllables</th>\n",
       "      <th>Words with 5 syllables</th>\n",
       "      <th>Words with 6 syllables</th>\n",
       "      <th>...</th>\n",
       "      <th>VB</th>\n",
       "      <th>VBD</th>\n",
       "      <th>VBG</th>\n",
       "      <th>VBN</th>\n",
       "      <th>VBP</th>\n",
       "      <th>VBZ</th>\n",
       "      <th>WDT</th>\n",
       "      <th>WP</th>\n",
       "      <th>WP$</th>\n",
       "      <th>WRB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>high</td>\n",
       "      <td>30</td>\n",
       "      <td>5.266667</td>\n",
       "      <td>5.303797</td>\n",
       "      <td>158</td>\n",
       "      <td>105</td>\n",
       "      <td>48</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>high</td>\n",
       "      <td>13</td>\n",
       "      <td>13.538462</td>\n",
       "      <td>5.403409</td>\n",
       "      <td>176</td>\n",
       "      <td>126</td>\n",
       "      <td>69</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>high</td>\n",
       "      <td>22</td>\n",
       "      <td>7.545455</td>\n",
       "      <td>5.620482</td>\n",
       "      <td>166</td>\n",
       "      <td>127</td>\n",
       "      <td>66</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>high</td>\n",
       "      <td>10</td>\n",
       "      <td>8.300000</td>\n",
       "      <td>5.939759</td>\n",
       "      <td>83</td>\n",
       "      <td>64</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>high</td>\n",
       "      <td>24</td>\n",
       "      <td>10.208333</td>\n",
       "      <td>6.979592</td>\n",
       "      <td>245</td>\n",
       "      <td>175</td>\n",
       "      <td>103</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25132</th>\n",
       "      <td>middle</td>\n",
       "      <td>13</td>\n",
       "      <td>4.846154</td>\n",
       "      <td>4.587302</td>\n",
       "      <td>63</td>\n",
       "      <td>36</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25133</th>\n",
       "      <td>middle</td>\n",
       "      <td>11</td>\n",
       "      <td>9.090909</td>\n",
       "      <td>5.150000</td>\n",
       "      <td>100</td>\n",
       "      <td>63</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25134</th>\n",
       "      <td>middle</td>\n",
       "      <td>20</td>\n",
       "      <td>4.300000</td>\n",
       "      <td>4.802326</td>\n",
       "      <td>86</td>\n",
       "      <td>57</td>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25135</th>\n",
       "      <td>middle</td>\n",
       "      <td>21</td>\n",
       "      <td>5.952381</td>\n",
       "      <td>5.048000</td>\n",
       "      <td>125</td>\n",
       "      <td>69</td>\n",
       "      <td>34</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25136</th>\n",
       "      <td>middle</td>\n",
       "      <td>18</td>\n",
       "      <td>5.111111</td>\n",
       "      <td>5.532609</td>\n",
       "      <td>92</td>\n",
       "      <td>63</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25137 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Class  Total Number Of Sentences  Average Sentence Length  \\\n",
       "0        high                         30                 5.266667   \n",
       "1        high                         13                13.538462   \n",
       "2        high                         22                 7.545455   \n",
       "3        high                         10                 8.300000   \n",
       "4        high                         24                10.208333   \n",
       "...       ...                        ...                      ...   \n",
       "25132  middle                         13                 4.846154   \n",
       "25133  middle                         11                 9.090909   \n",
       "25134  middle                         20                 4.300000   \n",
       "25135  middle                         21                 5.952381   \n",
       "25136  middle                         18                 5.111111   \n",
       "\n",
       "       Average Word Length  Number of Uncommon Words  Number of Unique Words  \\\n",
       "0                 5.303797                       158                     105   \n",
       "1                 5.403409                       176                     126   \n",
       "2                 5.620482                       166                     127   \n",
       "3                 5.939759                        83                      64   \n",
       "4                 6.979592                       245                     175   \n",
       "...                    ...                       ...                     ...   \n",
       "25132             4.587302                        63                      36   \n",
       "25133             5.150000                       100                      63   \n",
       "25134             4.802326                        86                      57   \n",
       "25135             5.048000                       125                      69   \n",
       "25136             5.532609                        92                      63   \n",
       "\n",
       "       Words with 1 to 3 syllables  Words with 4 syllables  \\\n",
       "0                               48                       4   \n",
       "1                               69                       3   \n",
       "2                               66                       6   \n",
       "3                               35                       3   \n",
       "4                              103                      18   \n",
       "...                            ...                     ...   \n",
       "25132                           10                       0   \n",
       "25133                           20                       2   \n",
       "25134                           21                       2   \n",
       "25135                           34                       2   \n",
       "25136                           26                       3   \n",
       "\n",
       "       Words with 5 syllables  Words with 6 syllables  ...  VB  VBD  VBG  VBN  \\\n",
       "0                           0                       0  ...   5    9    7    4   \n",
       "1                           0                       0  ...   5   13    7    6   \n",
       "2                           1                       0  ...   9   10    8    5   \n",
       "3                           0                       0  ...   2    4    1    4   \n",
       "4                           6                       0  ...   7    6    8    7   \n",
       "...                       ...                     ...  ...  ..  ...  ...  ...   \n",
       "25132                       0                       0  ...   0    8    0    0   \n",
       "25133                       1                       0  ...   5    1    2    2   \n",
       "25134                       0                       0  ...   2    1    2    0   \n",
       "25135                       0                       0  ...   1    0    5    0   \n",
       "25136                       0                       0  ...   1    1    4    0   \n",
       "\n",
       "       VBP  VBZ  WDT  WP  WP$  WRB  \n",
       "0        7    1    0   0    0    0  \n",
       "1        5    1    0   0    0    0  \n",
       "2        3    3    1   0    0    0  \n",
       "3        8    3    0   0    0    0  \n",
       "4       11    4    0   1    0    0  \n",
       "...    ...  ...  ...  ..  ...  ...  \n",
       "25132    0    0    0   0    0    0  \n",
       "25133    4    4    0   0    0    0  \n",
       "25134    2    6    0   0    0    0  \n",
       "25135    4    3    0   0    0    0  \n",
       "25136    4    3    0   0    0    0  \n",
       "\n",
       "[25137 rows x 47 columns]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_features.to_csv(\"./data/train_features.csv\", encoding=\"utf-8\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./data/test_features.csv\") as f:\n",
    "    ceshi = pd.read_csv(f,encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Total Number Of Sentences</th>\n",
       "      <th>Average Sentence Length</th>\n",
       "      <th>Average Word Length</th>\n",
       "      <th>Number of Uncommon Words</th>\n",
       "      <th>Number of Unique Words</th>\n",
       "      <th>Words with 1 to 3 syllables</th>\n",
       "      <th>Words with 4 syllables</th>\n",
       "      <th>Words with 5 syllables</th>\n",
       "      <th>Words with 6 syllables</th>\n",
       "      <th>Words with more than 7 syllables</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>high</td>\n",
       "      <td>42</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>231</td>\n",
       "      <td>173</td>\n",
       "      <td>81</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>high</td>\n",
       "      <td>45</td>\n",
       "      <td>7.622222</td>\n",
       "      <td>5.253644</td>\n",
       "      <td>343</td>\n",
       "      <td>231</td>\n",
       "      <td>100</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>high</td>\n",
       "      <td>24</td>\n",
       "      <td>6.708333</td>\n",
       "      <td>6.279503</td>\n",
       "      <td>161</td>\n",
       "      <td>124</td>\n",
       "      <td>58</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>high</td>\n",
       "      <td>35</td>\n",
       "      <td>7.571429</td>\n",
       "      <td>5.418868</td>\n",
       "      <td>265</td>\n",
       "      <td>193</td>\n",
       "      <td>99</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>high</td>\n",
       "      <td>22</td>\n",
       "      <td>7.545455</td>\n",
       "      <td>6.373494</td>\n",
       "      <td>166</td>\n",
       "      <td>115</td>\n",
       "      <td>65</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1402</th>\n",
       "      <td>middle</td>\n",
       "      <td>6</td>\n",
       "      <td>7.500000</td>\n",
       "      <td>5.066667</td>\n",
       "      <td>45</td>\n",
       "      <td>41</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1403</th>\n",
       "      <td>middle</td>\n",
       "      <td>28</td>\n",
       "      <td>4.785714</td>\n",
       "      <td>5.007463</td>\n",
       "      <td>134</td>\n",
       "      <td>88</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1404</th>\n",
       "      <td>middle</td>\n",
       "      <td>13</td>\n",
       "      <td>5.846154</td>\n",
       "      <td>5.421053</td>\n",
       "      <td>76</td>\n",
       "      <td>46</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1405</th>\n",
       "      <td>middle</td>\n",
       "      <td>18</td>\n",
       "      <td>4.111111</td>\n",
       "      <td>4.851351</td>\n",
       "      <td>74</td>\n",
       "      <td>50</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1406</th>\n",
       "      <td>middle</td>\n",
       "      <td>6</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>5.034483</td>\n",
       "      <td>29</td>\n",
       "      <td>19</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1407 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Class  Total Number Of Sentences  Average Sentence Length  \\\n",
       "0       high                         42                 5.500000   \n",
       "1       high                         45                 7.622222   \n",
       "2       high                         24                 6.708333   \n",
       "3       high                         35                 7.571429   \n",
       "4       high                         22                 7.545455   \n",
       "...      ...                        ...                      ...   \n",
       "1402  middle                          6                 7.500000   \n",
       "1403  middle                         28                 4.785714   \n",
       "1404  middle                         13                 5.846154   \n",
       "1405  middle                         18                 4.111111   \n",
       "1406  middle                          6                 4.833333   \n",
       "\n",
       "      Average Word Length  Number of Uncommon Words  Number of Unique Words  \\\n",
       "0                5.454545                       231                     173   \n",
       "1                5.253644                       343                     231   \n",
       "2                6.279503                       161                     124   \n",
       "3                5.418868                       265                     193   \n",
       "4                6.373494                       166                     115   \n",
       "...                   ...                       ...                     ...   \n",
       "1402             5.066667                        45                      41   \n",
       "1403             5.007463                       134                      88   \n",
       "1404             5.421053                        76                      46   \n",
       "1405             4.851351                        74                      50   \n",
       "1406             5.034483                        29                      19   \n",
       "\n",
       "      Words with 1 to 3 syllables  Words with 4 syllables  \\\n",
       "0                              81                       7   \n",
       "1                             100                      11   \n",
       "2                              58                       6   \n",
       "3                              99                       4   \n",
       "4                              65                       7   \n",
       "...                           ...                     ...   \n",
       "1402                           16                       1   \n",
       "1403                           26                       1   \n",
       "1404                           24                       2   \n",
       "1405                           20                       1   \n",
       "1406                            8                       0   \n",
       "\n",
       "      Words with 5 syllables  Words with 6 syllables  \\\n",
       "0                          1                       0   \n",
       "1                          2                       0   \n",
       "2                          2                       0   \n",
       "3                          1                       0   \n",
       "4                          6                       0   \n",
       "...                      ...                     ...   \n",
       "1402                       0                       0   \n",
       "1403                       0                       0   \n",
       "1404                       0                       0   \n",
       "1405                       0                       0   \n",
       "1406                       0                       0   \n",
       "\n",
       "      Words with more than 7 syllables  \n",
       "0                                   84  \n",
       "1                                  118  \n",
       "2                                   58  \n",
       "3                                   89  \n",
       "4                                   37  \n",
       "...                                ...  \n",
       "1402                                24  \n",
       "1403                                61  \n",
       "1404                                20  \n",
       "1405                                29  \n",
       "1406                                11  \n",
       "\n",
       "[1407 rows x 11 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ceshi.iloc[:,0:11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py_torch",
   "language": "python",
   "name": "py_torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
